<!DOCTYPE html>
<html lang="en">

<!-- Head tag -->
<head><meta name="generator" content="Hexo 3.8.0">

    <meta charset="utf-8">
    <meta http-equiv="X-UA-Compatible" content="IE=edge">
    <meta name="viewport" content="width=device-width, initial-scale=1">

    <!--Description-->
    
        <meta name="description" content="欢迎！">
    

    <!--Author-->
    
        <meta name="author" content="fafa">
    

    <!--Open Graph Title-->
    
        <meta property="og:title" content="Run-Fa Zhang&#39;s Site">
    

    <!--Open Graph Description-->
    
        <meta property="og:description" content="欢迎！">
    

    <!--Open Graph Site Name-->
    <meta property="og:site_name" content="Run-Fa Zhang&#39;s Site">

    <!--Type page-->
    
        <meta property="og:type" content="website">
    

    <!--Page Cover-->
    

    <meta name="twitter:card" content="summary">
    

    <!-- Title -->
    
    <title>Run-Fa Zhang&#39;s Site</title>

    <!-- Bootstrap Core CSS -->
    <link rel="stylesheet" href="//maxcdn.bootstrapcdn.com/bootstrap/4.0.0-alpha.2/css/bootstrap.min.css" integrity="sha384-y3tfxAZXuh4HwSYylfB+J125MxIs6mR5FOHamPBG064zB+AFeWH94NdvaCBm8qnd" crossorigin="anonymous">

    <!-- Custom Fonts -->
    <link href="//maxcdn.bootstrapcdn.com/font-awesome/4.6.3/css/font-awesome.min.css" rel="stylesheet" type="text/css">

    <!-- HTML5 Shim and Respond.js IE8 support of HTML5 elements and media queries -->
    <!-- WARNING: Respond.js doesn't work if you view the page via file:// -->
    <!--[if lt IE 9]>
        <script src="//oss.maxcdn.com/libs/html5shiv/3.7.0/html5shiv.js"></script>
        <script src="//oss.maxcdn.com/libs/respond.js/1.4.2/respond.min.js"></script>
    <![endif]-->

    <!-- Gallery -->
    <link href="//cdnjs.cloudflare.com/ajax/libs/featherlight/1.3.5/featherlight.min.css" type="text/css" rel="stylesheet">

    <!-- Custom CSS -->
    <link rel="stylesheet" href="/css/style.css">

    <!-- Google Analytics -->
    


</head>


<body>

<div class="bg-gradient"></div>
<div class="bg-pattern"></div>

<!-- Menu -->
<!--Menu Links and Overlay-->
<div class="menu-bg">
    <div class="menu-container">
        <ul>
            
            <li class="menu-item">
                <a href="/">
                    Home
                </a>
            </li>
            
            <li class="menu-item">
                <a href="/archives">
                    Archives
                </a>
            </li>
            
            <li class="menu-item">
                <a href="/about.html">
                    About
                </a>
            </li>
            
            <li class="menu-item">
                <a href="/tags">
                    Tags
                </a>
            </li>
            
            <li class="menu-item">
                <a href="/categories">
                    Categories
                </a>
            </li>
            
            <li class="menu-item">
                <a href="/contact.html">
                    Contact
                </a>
            </li>
            
        </ul>
    </div>
</div>

<!--Hamburger Icon-->
<nav>
    <a href="#menu"></a>
</nav>

<div class="container">

    <!-- Main Content -->
    <div class="row">
    <div class="col-sm-12">

        <!--Title and Logo-->
        <header>
    <div class="logo">
        <a href="/"><i class="logo-icon fa fa-cube" aria-hidden="true"></i></a>
        
    </div>
</header>

        <section class="main">
            
<div class="post">

    <div class="post-header">
        <h1 class="title">
            <a href="/2019/05/20/user-17163699-1558319415/Python/数据预处理之——腾讯广告算法大赛/">
                Untitled
            </a>
        </h1>
        <div class="post-info">
            
                <span class="date">2019-05-20</span>
            
            
            
        </div>
    </div>

    <div class="content">

        <!-- Gallery -->
        

        <!-- Post Content -->
        <!DOCTYPE html>
<html lang="en">
  <head><meta name="generator" content="Hexo 3.8.0">
    <meta charset="utf-8">
    <meta name="viewport" content="width=device-width, initial-scale=1.0">
    <title>数据预处理之——腾讯广告算法大赛</title>
    <style type="text/css" media="all">
      body {
        margin: 0;
        font-family: "Helvetica Neue", Helvetica, Arial, "Hiragino Sans GB", sans-serif;
        font-size: 14px;
        line-height: 20px;
        color: #777;
        background-color: white;
      }
      .container {
        width: 700px;
        margin-right: auto;
        margin-left: auto;
      }

      .post {
        font-family: Georgia, "Times New Roman", Times, "SimSun", serif;
        position: relative;
        padding: 70px;
        bottom: 0;
        overflow-y: auto;
        font-size: 16px;
        font-weight: normal;
        line-height: 25px;
        color: #515151;
      }

      .post h1{
        font-size: 50px;
        font-weight: 500;
        line-height: 60px;
        margin-bottom: 40px;
        color: inherit;
      }

      .post p {
        margin: 0 0 35px 0;
      }

      .post img {
        border: 1px solid #D9D9D9;
      }

      .post a {
        color: #28A1C5;
      }
    </style>
  </head>
  <body>
    <div class="container">
      <div class="post">
        <h1 class="title">数据预处理之——腾讯广告算法大赛</h1>
        <div class="show-content">
          <p>```</p><p></p><p>#首先了解如何快速做一个列表﻿</p><p></p><p>df4 = pd.DataFrame({'col1':['1',3],'col2':[2,4]},index=['a','b'])﻿</p><p>#接下来我们来看如何处理脏数据﻿</p><p>import pandas as pd﻿</p><p>import numpy as np﻿</p><p>df=pd.DataFrame({"id":["1","2","3,4"]})﻿</p><p>df﻿﻿<br></p><p>def max_str(t):﻿</p><p>    a=[int(i) for i in t]﻿</p><p>    return max(a)﻿</p><p>df["id_max"]=df["id"].str.split(",").map(max_str)﻿</p><p>df﻿﻿<br></p><p>#腾讯广告算法大赛为例子﻿</p><p>##################################制作小样本集##############################﻿</p><p>###########################################################################﻿</p><p>#第一步 读取数据﻿</p><p>df=pd.read_table('C:/Users/fafa/Desktop/testA/user_data',sep = '\t',header=None,engine='python')﻿</p><p>#增加 列名﻿</p><p>df.columns=['用户ID','年龄','性别','地域','婚恋状态','学历','消费能力','设备','工作状态','连接类型','行为性趣']﻿</p><p>#切分数据，使用前2000条﻿</p><p>df2=df.head(2000)﻿</p><p>#导出切分好的数据﻿</p><p>df2.to_excel('C:/Users/fafa/Desktop/testa/user_data.xls')﻿﻿<br></p><p>##################同理，制作 其余的小样本集##########﻿</p><p>test=pd.read_table('C:/Users/fafa/Desktop/testA/test_sample.dat',sep = '\t',header=None,engine='python')﻿</p><p>test.columns=['样本id','广告id','创建时间','素材尺寸','广告行业id','商品类型','商品id','广告账户id','投放时段','人群定向','出价(单位分)']﻿</p><p>test.to_excel('C:/Users/fafa/Desktop/testa/测试数据.xls')﻿﻿<br></p><p>test=pd.read_table('C:/Users/fafa/Desktop/testA/ad_operation.dat',sep = '\t',header=None,engine='python')﻿</p><p>test.columns=['广告id','创建/修改时间','操作类型','修改字段','操作后的字段']﻿</p><p>df3=test.head(2000)﻿</p><p>df3.to_excel('C:/Users/fafa/Desktop/testa/广告操作数据.xls')﻿﻿<br></p><p>df4=pd.read_table('C:/Users/fafa/Desktop/testA/ad_static_feature.out',sep = '\t',header=None,engine='python')﻿</p><p>df4.columns=['广告id','创建时间','广告账户id','商品id','商品类型','广告行业id','素材尺寸']﻿</p><p>df5=df4.head(2000)﻿</p><p>df5.to_excel('C:/Users/fafa/Desktop/testa/广告静态数据.xls')﻿﻿<br></p><p>df=pd.read_table('C:/Users/fafa/Desktop/testA/想',sep = '\t',header=None,engine='python')﻿﻿<br></p><p>df.columns=['广告请求id','广告请求时间','广告位id','用户id','曝光广告id','曝光广告素材尺寸','曝光广告出价bid','曝光广告pctr','曝光广告quality_ecpm','曝光广告totalEcpm']﻿﻿<br></p><p>df=df.head(2000)﻿﻿<br></p><p>df.to_excel('C:/Users/fafa/Desktop/广告曝光日志.xls')﻿﻿<br></p><p>###################################################﻿</p><p>###################################################﻿</p><p>##########统计日志中广告id的出现次数，并关联其信息##########﻿</p><p>###################################################﻿</p><p>###################################################﻿</p><p>#读取曝光日志﻿</p><p>df=pd.read_excel('D:/mini数据集/曝光日志.xls',header=0)﻿</p><p>#对"姓名" 计数，得到 “姓名”和“计数”两列。﻿</p><p>df.姓名.value_counts().to_frame().reset_index().rename({"index":"姓名","姓名":"计数"},axis=1)﻿﻿<br></p><p>#对"曝光广告id" 计数，得到 “广告id”和“曝光次数”两列。﻿</p><p>df1=df.曝光广告id.value_counts().to_frame().reset_index().rename({"index":"广告id","曝光广告id":"曝光次数"},axis=1)df1.head()﻿</p><p>#法二 ﻿</p><p># df['count'] = 1 ﻿</p><p>#df.groupby('Name')['count'].agg('sum')﻿﻿<br></p><p>#关联广告其他属性 到 曝光次数 表格﻿</p><p>#读取广告其他属性df2﻿</p><p>df2=pd.read_excel('D:/mini数据集/广告静态数据.xls',header=0)df2.head()﻿</p><p>#关联﻿</p><p>df1.merge(df2,on="广告id")﻿</p><p>#删除不需要的列﻿</p><p>df4=df3.drop({'创建时间',"广告账户id"},1)df4.head()﻿</p><p>#修改列的位置﻿</p><p>df=df[['广告id', '商品id', '商品类型', '广告行业id', '素材尺寸','曝光次数']]﻿﻿<br></p><p>#################################################﻿</p><p>###########以下是直接调取大形数据制作训练集#########﻿</p><p>#################################################﻿</p><p>#第一步 读取数据df1=pd.read_table('C:/Users/fafa/Desktop/testA/totalExposureLog.out',sep = '\t',header=None,engine='python')df2=pd.read_table('C:/Users/fafa/Desktop/testA/ad_static_feature.out',sep = '\t',header=None,engine='python')﻿</p><p>#增加 列名﻿</p><p>df1.columns=['广告请求id','广告请求时间','广告位id','用户id','曝光广告id','曝光广告素材尺寸','曝光广告出价bid','曝光广告pctr','曝光广告quality_ecpm','曝光广告totalEcpm']df2.columns=['广告id','创建时间','广告账户id','商品id','商品类型','广告行业id','素材尺寸']#对"曝光广告id" 计数，得到 “广告id”和“曝光次数”两列。 ﻿</p><p>df3=df1.曝光广告id.value_counts().to_frame().reset_index().rename({"index":"广告id","曝光广告id":"曝光次数"},axis=1) df1.head() ﻿</p><p>#关联广告其他属性 到 曝光次数 表格 ﻿</p><p>df3.merge(df2,on="广告id") ﻿</p><p>#删除不需要的列 ﻿</p><p>df4=df3.drop({'创建时间',"广告账户id"},1) df4.head() ﻿</p><p>#修改列的位置 ﻿</p><p>df5=df4[['广告id', '商品id', '商品类型', '广告行业id', '素材尺寸','曝光次数']] ﻿</p><p>#发现id中存在脏数据 所以清理一下（方法见脏数据的清理）﻿</p><p>def max_str(t):﻿</p><p>    a=[int(i) for i in t] ﻿</p><p>    return max(a)﻿</p><p>df5["广告id"]=df5["广告id"].str.split(",").map(max_str)﻿</p><p>df5["商品id"]=df5["商品id"].str.split(",").map(max_str)﻿</p><p>df5["广告行业id"]=df5["广告行业id"].str.split(",").map(max_str)﻿</p><p>#令空值NaN为0﻿</p><p>df5.fillna(0)﻿</p><p>df5.head()﻿﻿<br></p><p>###最终得到的df5就是一个数据集合，最后一列是Y，其余列都是特征X（要注意这里df5是有列名的）。然后套用nn模板(或者Light gbm 代码在此)来训练即可。﻿﻿<br></p><p>####nn模板如下####﻿</p><p># -*- coding: utf-8 -*-﻿</p><p>import pandas as pd﻿</p><p>import numpy as np﻿</p><p>from keras import metrics﻿</p><p>from keras.models import Sequential﻿</p><p>from keras.layers import Dense﻿</p><p>from keras.wrappers.scikit_learn import KerasClassifier﻿</p><p>from sklearn.model_selection import KFold, cross_val_scoredataset=pd.read_csv('housing.csv',header=None)﻿</p><p>X=dataset.iloc[:,0:13]﻿</p><p>Y=dataset.iloc[:,13]﻿</p><p># print(Y)﻿</p><p>seed=7﻿</p><p>np.random.seed(seed)﻿</p><p># 建立模型﻿</p><p>optimizer='adam'﻿</p><p>init='normal'﻿</p><p>model=Sequential()﻿</p><p>model.add(Dense(units=13,activation='relu',input_dim=13,kernel_initializer=init))﻿</p><p>#构建更多的隐藏层﻿</p><p>model.add(Dense(units=10,activation='relu',kernel_initializer=init))﻿</p><p>model.add(Dense(units=1,kernel_initializer=init)) ﻿</p><p>#输出层不需要进行激活函数,预测回归的话unit=1# 编译模型﻿</p><p>model.compile(loss='mse',optimizer=optimizer,metrics=['acc'])﻿</p><p>model.fit(X.values,Y.values,epochs=30,batch_size=64)﻿﻿<br></p><p>```</p><div class="image-package">
<img name src="http://upload-images.jianshu.io/upload_images/17163699-7e8b5104317ce3a2.png?imageMogr2/auto-orient/strip%7CimageView2/2/w/1240"><br><div class="image-caption"></div>
</div><p><a href="https://zhuanlan.zhihu.com/p/34634023" target="_blank">数据分析基本过程</a></p><p><a href="https://blog.csdn.net/qq_41776781/article/details/89873430" target="_blank">XGBOOST模型训练数据集</a><br></p><hr id="null"><p>If you are interested in this topic.﻿<br>You can get in touch with me.<br>18234056952(Tel  wechat  qq)</p>
        </div>
      </div>
    </div>
  </body>
</html>

    </div>

    

    

    <!-- Comments -->
    

</div>
        </section>

    </div>
</div>


</div>

<!-- Footer -->
<div class="push"></div>

<footer class="footer-content">
    <div class="container">
        <div class="row">
            <div class="col-xs-12 col-sm-12 col-md-6 col-lg-6 footer-about">
                <h2>About</h2>
                <p>
                    This theme was developed by <a href="https://github.com/klugjo">Jonathan Klughertz</a>. The source code is available on Github. Create Websites. Make Magic.
                </p>
            </div>
            
    <div class="col-xs-6 col-sm-6 col-md-3 col-lg-3 recent-posts">
        <h2>Recent Posts</h2>
        <ul>
            
            <li>
                <a class="footer-post" href="/2019/05/20/user-17163699-1558319415/量子科学/量子算法/">Untitled</a>
            </li>
            
            <li>
                <a class="footer-post" href="/2019/05/20/user-17163699-1558319415/数学建模/建模/">Untitled</a>
            </li>
            
            <li>
                <a class="footer-post" href="/2019/05/20/user-17163699-1558319415/操作系统/Win10自带linux子系统/">Untitled</a>
            </li>
            
            <li>
                <a class="footer-post" href="/2019/05/20/user-17163699-1558319415/其他/Markdown/">Untitled</a>
            </li>
            
        </ul>
    </div>



            
        </div>
        <div class="row">
            <div class="col-xs-12 col-sm-12 col-md-12 col-lg-12">
                <ul class="list-inline footer-social-icons">
                    
                    <li class="list-inline-item">
                        <a href="https://github.com/klugjo/hexo-theme-alpha-dust">
                            <span class="footer-icon-container">
                                <i class="fa fa-github"></i>
                            </span>
                        </a>
                    </li>
                    
                    
                    <li class="list-inline-item">
                        <a href="https://twitter.com/?lang=en">
                            <span class="footer-icon-container">
                                <i class="fa fa-twitter"></i>
                            </span>
                        </a>
                    </li>
                    
                    
                    <li class="list-inline-item">
                        <a href="https://www.facebook.com/">
                            <span class="footer-icon-container">
                                <i class="fa fa-facebook"></i>
                            </span>
                        </a>
                    </li>
                    
                    
                    <li class="list-inline-item">
                        <a href="https://www.instagram.com/">
                            <span class="footer-icon-container">
                                <i class="fa fa-instagram"></i>
                            </span>
                        </a>
                    </li>
                    
                    
                    <li class="list-inline-item">
                        <a href="https://dribbble.com/">
                            <span class="footer-icon-container">
                                <i class="fa fa-dribbble"></i>
                            </span>
                        </a>
                    </li>
                    
                    
                    <li class="list-inline-item">
                        <a href="https://plus.google.com/">
                            <span class="footer-icon-container">
                                <i class="fa fa-google-plus"></i>
                            </span>
                        </a>
                    </li>
                    
                    
                    <li class="list-inline-item">
                        <a href="https://www.behance.net/">
                            <span class="footer-icon-container">
                                <i class="fa fa-behance"></i>
                            </span>
                        </a>
                    </li>
                    
                    
                    <li class="list-inline-item">
                        <a href="https://500px.com/">
                            <span class="footer-icon-container">
                                <i class="fa fa-500px"></i>
                            </span>
                        </a>
                    </li>
                    
                    
                    <li class="list-inline-item">
                        <a href="mailto:test@example.com">
                            <span class="footer-icon-container">
                                <i class="fa fa-envelope-o"></i>
                            </span>
                        </a>
                    </li>
                    
                    
                    <li class="list-inline-item">
                        <a href="\#">
                            <span class="footer-icon-container">
                                <i class="fa fa-rss"></i>
                            </span>
                        </a>
                    </li>
                    
                </ul>
            </div>
        </div>
        <div class="row">
            <div class="col-xs-12 col-sm-12 col-md-12 col-lg-12">
                <div class="footer-copyright">
                    @Untitled. All right reserved | Design & Hexo <a href="http://www.codeblocq.com/">Jonathan Klughertz</a>
                </div>
            </div>
        </div>
    </div>
</footer>

<!-- After footer scripts -->

<!-- jQuery -->
<script src="//code.jquery.com/jquery-2.1.4.min.js"></script>

<!-- Tween Max -->
<script src="//cdnjs.cloudflare.com/ajax/libs/gsap/1.18.5/TweenMax.min.js"></script>

<!-- Gallery -->
<script src="//cdnjs.cloudflare.com/ajax/libs/featherlight/1.3.5/featherlight.min.js" type="text/javascript" charset="utf-8"></script>

<!-- Custom JavaScript -->
<script src="/js/main.js"></script>

<!-- Disqus Comments -->



</body>

</html>